/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

package org.tensorflow.lite.benchmark.delegateperformance;

/** Enumerates the possible benchmark result values. */
public enum BenchmarkResultType {
  // Unknown benchmark result, possibly due to internal failures.
  UNKNOWN("UNKNOWN"),
  // The benchmark results are not involved in the Pass/Pass with Warning/Fail result generation.
  NOT_APPLICABLE("N/A"),
  // The benchmark activity skips the Pass/Pass with Warning/Fail result generation.
  SKIP("SKIP"),

  // The meanings of "PASS", "PASS_WITH_WARNING" and "FAIL":
  //
  // Latency benchmark:
  // 1. When the test target delegate type is the same as the reference delegate:
  // the test target delegate is expected to be better, or at least not
  // substantially worse, than the reference delegate in ALL metrics.
  //    - PASS: All performance metrics of the test target delegate are better
  //    than or equal to the reference delegate.
  //    - PASS_WITH_WARNING: At least 1 metric is worse than the reference
  //    delegate, but not substantially worse. No regression thresholds are
  //    breached.
  //    - FAIL: At least 1 regression threshold is breached.
  // 2. When the test target delegate type is different from the reference
  // delegate: the test delegate is only expected to be better, or at least not
  // substantially worse, than the reference delegate in AT LEAST 1 metric.
  //    - PASS: All performance metrics of the test target delegate are better than
  //    or equal to the reference delegate.
  //    - PASS_WITH_WARNING: At least 1 metric is worse than the reference
  //    delegate, but, at least 1 regression threshold is not breached.
  //    - FAIL: All regression thresholds are breached.
  // 3. The overall result is generated by aggregating the results from the pairs.
  //    - PASS: All results are "PASS".
  //    - PASS_WITH_WARNING: At least 1 "PASS_WITH_WARNING" result and 0 "FAIL"
  //    result.
  //    - FAIL: At least 1 "FAIL" result.
  //
  // Accuracy benchmark:
  // The accuracy benchmark delegates the accuracy metric threshold checks to the metric scripts,
  // which are embedded together with the test input and the ground truth inside the models. The
  // metric scripts are responsible for computing the accuracy metrics (e.g. Mean square error) from
  // the model predictions and the ground truth data, checking the accuracy metrics against the
  // pre-defined thresholds and aggregating the results into a binarized pass status.
  // 1. When the metric scripts check the outcomes for every model and every delegate:
  //    - PASS: No accuracy threshold in the model is breached by the delegate.
  //    - FAIL: At least 1 accuracy threshold threshold is breached.
  // 2. The overall result is generated by aggregating the results from the delegate results for
  // each model.
  //    - PASS: The test target delegate result is "PASS".
  //    - PASS_WITH_WARNING: The test delegate and all reference delegates results are "FAIL".
  //    - FAIL: The test target delegate result is "FAIL" and at least 1 reference delegate result
  //    is "PASS".
  PASS("PASS"),
  PASS_WITH_WARNING("PASS_WITH_WARNING"),
  FAIL("FAIL");

  private final String name;

  BenchmarkResultType(String name) {
    this.name = name;
  }

  @Override
  public String toString() {
    return name;
  }
}
